{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dpp\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.distributions as td\n",
    "from copy import deepcopy\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_style('whitegrid')\n",
    "torch.set_default_tensor_type(torch.cuda.FloatTensor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Config\n",
    "\n",
    "Change the values bellow to train on other datasets / with other models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 1\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "\n",
    "## General data config\n",
    "dataset_name = 'synth/hawkes2' # other: [ 'yelp_toronto', 'wikipedia', 'mooc', 'stack_overflow', 'lastfm',  \n",
    "                               #          'reddit', 'synth/poisson', 'synth/renewal', 'synth/self_correcting', \n",
    "                               #          'synth/hawkes1', 'synth/hawkes2']\n",
    "\n",
    "split = 'whole_sequences' # How to split the sequences (other 'each_sequence' -- split every seq. into train/val/test)\n",
    "\n",
    "## General model config\n",
    "use_history = True        # Whether to use RNN to encode history\n",
    "history_size = 64         # Size of the RNN hidden vector\n",
    "rnn_type = 'RNN'          # Which RNN cell to use (other: ['GRU', 'LSTM'])\n",
    "use_embedding = False     # Whether to use sequence embedding (should use with 'each_sequence' split)\n",
    "embedding_size = 32       # Size of the sequence embedding vector\n",
    "                          # IMPORTANT: when using split = 'whole_sequences', the model will only learn embeddings\n",
    "                          # for the training sequences, and not for validation / test\n",
    "trainable_affine = False  # Train the final affine layer\n",
    "\n",
    "## Decoder config\n",
    "decoder_name = 'LogNormMix' # other: ['RMTPP', 'FullyNeuralNet', 'Exponential', 'SOSPolynomial', 'DeepSigmoidalFlow']\n",
    "n_components = 64           # Number of components for a mixture model\n",
    "hypernet_hidden_sizes = []  # Number of units in MLP generating parameters ([] -- affine layer, [64] -- one layer, etc.)\n",
    "\n",
    "## Flow params\n",
    "# Polynomial\n",
    "max_degree = 3  # Maximum degree value for Sum-of-squares polynomial flow (SOS)\n",
    "n_terms = 4     # Number of terms for SOS flow\n",
    "# DSF / FullyNN\n",
    "n_layers = 2    # Number of layers for Deep Sigmoidal Flow (DSF) / Fully Neural Network flow (Omi et al., 2019)\n",
    "layer_size = 64 # Number of mixture components / units in a layer for DSF and FullyNN\n",
    "\n",
    "## Training config\n",
    "regularization = 1e-5 # L2 regularization parameter\n",
    "learning_rate = 1e-3  # Learning rate for Adam optimizer\n",
    "max_epochs = 1000     # For how many epochs to train\n",
    "display_step = 50     # Display training statistics after every display_step\n",
    "patience = 50         # After how many consecutive epochs without improvement of val loss to stop training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data\n",
    "\n",
    "- Load dataset\n",
    "- Split into training / validation / test set\n",
    "- Normalize input inter-event times\n",
    "- Break down long traning set sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if '+' not in dataset_name:\n",
    "    dataset = dpp.data.load_dataset(dataset_name)\n",
    "else:\n",
    "    # If '+' in dataset_name, load all the datasets together and concatenate them\n",
    "    # For example, dataset_name='synth/poisson+synth/renewal' loads poisson and renewal datasets\n",
    "    dataset_names = [d.strip() for d in dataset_name.split('+')]\n",
    "    dataset = dpp.data.load_dataset(dataset_names.pop(0))\n",
    "    for d in dataset_names:\n",
    "        dataset += dpp.data.load_dataset(dataset_names.pop(0))\n",
    "\n",
    "# Split into train/val/test, on each sequence or assign whole sequences to different sets\n",
    "if split == 'each_sequence':\n",
    "    d_train, d_val, d_test = dataset.train_val_test_split_each(seed=seed)\n",
    "elif split == 'whole_sequences':\n",
    "    d_train, d_val, d_test = dataset.train_val_test_split_whole(seed=seed)\n",
    "else:\n",
    "    raise ValueError(f'Unsupported dataset split {split}')\n",
    "\n",
    "# Calculate mean and std of the input inter-event times and normalize only input\n",
    "mean_in_train, std_in_train = d_train.get_mean_std_in()\n",
    "std_out_train = 1.0\n",
    "d_train.normalize(mean_in_train, std_in_train, std_out_train)\n",
    "d_val.normalize(mean_in_train, std_in_train, std_out_train)\n",
    "d_test.normalize(mean_in_train, std_in_train, std_out_train)\n",
    "\n",
    "# Break down long train sequences for faster batch traning and create torch DataLoaders\n",
    "d_train.break_down_long_sequences(128)\n",
    "collate = dpp.data.collate\n",
    "dl_train = torch.utils.data.DataLoader(d_train, batch_size=64, shuffle=True, collate_fn=collate)\n",
    "dl_val = torch.utils.data.DataLoader(d_val, batch_size=1, shuffle=False, collate_fn=collate)\n",
    "dl_test = torch.utils.data.DataLoader(d_test, batch_size=1, shuffle=False, collate_fn=collate)\n",
    "\n",
    "# Set the parameters for affine normalization layer depending on the decoder (see Appendix D.3 in the paper)\n",
    "if decoder_name in ['RMTPP', 'FullyNeuralNet', 'Exponential']:\n",
    "    _, std_out_train = d_train.get_mean_std_out()\n",
    "    mean_out_train = 0.0\n",
    "else:\n",
    "    mean_out_train, std_out_train = d_train.get_log_mean_std_out()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model setup\n",
    "\n",
    "- Define the model config\n",
    "- Define the optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# General model config\n",
    "general_config = dpp.model.ModelConfig(\n",
    "    use_history=use_history,\n",
    "    history_size=history_size,\n",
    "    rnn_type=rnn_type,\n",
    "    use_embedding=use_embedding,\n",
    "    embedding_size=embedding_size,\n",
    "    num_embeddings=len(dataset),\n",
    ")\n",
    "\n",
    "# Decoder specific config\n",
    "decoder = getattr(dpp.decoders, decoder_name)(general_config,\n",
    "                                              n_components=n_components,\n",
    "                                              hypernet_hidden_sizes=hypernet_hidden_sizes,\n",
    "                                              max_degree=max_degree,\n",
    "                                              n_terms=n_terms,\n",
    "                                              n_layers=n_layers,\n",
    "                                              layer_size=layer_size,\n",
    "                                              shift_init=mean_out_train,\n",
    "                                              scale_init=std_out_train,\n",
    "                                              trainable_affine=trainable_affine)\n",
    "\n",
    "# Define model\n",
    "model = dpp.model.Model(general_config, decoder)\n",
    "model.use_history(general_config.use_history)\n",
    "model.use_embedding(general_config.use_embedding)\n",
    "\n",
    "# Define optimizer\n",
    "opt = torch.optim.Adam(model.parameters(), weight_decay=regularization, lr=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Traning\n",
    "\n",
    "- Run for max_epochs or until the early stopping condition is satisfied\n",
    "- Calculate and save the training statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function that calculates the loss for the entire dataloader\n",
    "def get_total_loss(loader):\n",
    "    loader_log_prob, loader_lengths = [], []\n",
    "    for input in loader:\n",
    "        loader_log_prob.append(model.log_prob(input).detach())\n",
    "        loader_lengths.append(input.length.detach())\n",
    "    return -model.aggregate(loader_log_prob, loader_lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch   50, loss_train_last_batch = 0.0382, loss_val = -0.0271\n",
      "Epoch  100, loss_train_last_batch = 0.0133, loss_val = -0.0292\n",
      "Breaking due to early stopping at epoch 139\n"
     ]
    }
   ],
   "source": [
    "impatient = 0\n",
    "best_loss = np.inf\n",
    "best_model = deepcopy(model.state_dict())\n",
    "training_val_losses = []\n",
    "\n",
    "for epoch in range(max_epochs):\n",
    "    model.train()\n",
    "    for input in dl_train:\n",
    "        opt.zero_grad()\n",
    "        log_prob = model.log_prob(input)\n",
    "        loss = -model.aggregate(log_prob, input.length)\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "\n",
    "    model.eval()\n",
    "    loss_val = get_total_loss(dl_val)\n",
    "    training_val_losses.append(loss_val.item())\n",
    "\n",
    "    if (best_loss - loss_val) < 1e-4:\n",
    "        impatient += 1\n",
    "        if loss_val < best_loss:\n",
    "            best_loss = loss_val.item()\n",
    "            best_model = deepcopy(model.state_dict())\n",
    "    else:\n",
    "        best_loss = loss_val.item()\n",
    "        best_model = deepcopy(model.state_dict())\n",
    "        impatient = 0\n",
    "\n",
    "    if impatient >= patience:\n",
    "        print(f'Breaking due to early stopping at epoch {epoch}')\n",
    "        break\n",
    "\n",
    "    if (epoch + 1) % display_step == 0:\n",
    "        print(f\"Epoch {epoch+1:4d}, loss_train_last_batch = {loss:.4f}, loss_val = {loss_val:.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation\n",
    "\n",
    "- Load the best model\n",
    "- Calculate the train/val/test loss\n",
    "- Plot the training curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time NLL\n",
      "Train: 0.0192\n",
      "Val:   -0.0299\n",
      "Test:  0.0657\n"
     ]
    }
   ],
   "source": [
    "model.load_state_dict(best_model)\n",
    "model.eval()\n",
    "\n",
    "pdf_loss_train = get_total_loss(dl_train)\n",
    "pdf_loss_val = get_total_loss(dl_val)\n",
    "pdf_loss_test = get_total_loss(dl_test)\n",
    "\n",
    "print(f'Time NLL\\n'\n",
    "      f'Train: {pdf_loss_train:.4f}\\n'\n",
    "      f'Val:   {pdf_loss_val.item():.4f}\\n' \n",
    "      f'Test:  {pdf_loss_test.item():.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEWCAYAAABIVsEJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU1dnA8d9km5AdEiCBsIMPILJvCi64VeuCdam7SLHW1lqrra22b231tX3V1uKudUGhLohWK9atigouiBAWEeLDvoQ9LAkhZJ/3j3uDk5VhyGSSzPP9fPJh7p0z9z453Nxnzjn3nuvx+XwYY4wxRyoq3AEYY4xpnSyBGGOMCYolEGOMMUGxBGKMMSYolkCMMcYExRKIMcaYoFgCMfUSkWgRKRKR7k1ZNlKJyHUi8skRlP9MRK4NXUTNs4+m0FrijEQx4Q7ANA0RKfJbTABKgUp3+Seq+uKRbE9VK4Gkpi4bTiLyGXA7cBZQoqr3hGg/fYHVquo5ws/FA1uAbqGIK9REJBN4EDgZ5xj8GrhVVRe6718HjAXuBd5T1b4hiCEPuEpVP2nqbYdjPy2dtUDaCFVNqv4BNgHn+a2rkzxExL48tDynAAtVtTjcgQQpCfgSGAZ0AF4C3haRhLBGZULGTiIRQkTuAfoBVcC5wE0iosBUoD9wEHgV+JWqlrsJphzopaobROQFYI+7jfHAN8AVqrr+SMq6sZwNPAR0BmYAw4GnVfX5euKOB+4HLnFjfwW4XVXLROR04BngceA2N4bbVXXGEdRLJ+B54AR3+9+o6kkicgcwVFUv9Sv7BHBAVX/ttmbmAGcAxwGfu7/jHmCeW766VTjB/dcjIlOByW793KCq//UL5/vAO37LvUTki9rbF5EoYBZO3cYDS4GfqmquiPTDOYlnqKpPRJ4HzlTVLm5MM4HPVPXRWvXQBfgv8IyqPigiaTjHxlk4LdlpwJ9UtUpEjnHrfYhb5/9V1StUdQ1OC6TaEyLyN5zjYFmj/xE1YzkL5/jIxPm/8fi91w94ChgM+IB3gZ+raoGIvAx0Ad4VkUrgTjeeeuvK3d65wF+BbKAAeEBVp7rvnQ/8L9AD5xi+QVW/qW8/qvr3QH+/tsRaIJHlBzjfClNxTsQVwM1ABjAO52Txk0Y+fwXwB5xvl5tw/riOqKx7wp6Fc8LPANYDoxvZzp3ASJwTxjA3zjv83s8G2uH8Qd+Ac9JKqW9DqjpeVT9T1f/x6766DVgHdMQ5Yf3BXf9P4JzqbYlIHE4S+2et33ESTiJMBG5115/k7q+6BbjQXX8CsBxIxzk5P1srxLOpmUAa2j7Af3BOzJk4J7d/uvtcDZS49QVwIlDqnnirY5vrv1MR6eOum6qq1QngBZwvFX1w6v8cnMQH8GfgbaA9Tv0/Rj1EZCTOyX+dG9szqnqdqq5pqPvKPT5ew+lqzADygDF+RTzAPUAWMBDojft/pqqXA1uBs916rz6p11tXrueAKaqa7NbZXDeOUcDTwHU4/1/TgDdFJK6R/UQca4FEls9U9S339UFgod9760TkKZz+60frfNLxmqouAhCRF4G/NLKvhsqeCyxV1Tfd96binMQbciXwY1Xd5Za/G+fb6V3u+yXAPe44zGwRKQWOARY1sk1/5Tgnye6quhb3BKKqeSIyH7gI5yTzfWCrqvp/k37WPWEjIq8CZx5mX2tVdZpbfjrwsIhkqGq+iAhQ5X6Lb3T7qlqF880c970/AbtEJFFVD+C0gE4WkXy3ft53lwG8OCfRaoOAPwG/UdVZ7va6AqcBaapaChSLyIPANThJrxzoCWSp6hac1lENIpIKTAf+qKr7D1Mv/qqPjzfc7TyAX+JU1VXAKndxp3v8/LahjQVQV+XAQBFZ7rYe97hFrwce90v+00Tk98Co+n7fSGUJJLJs9l8Qkf7AA8AInEHPGGBBI5/f7ve6mMYHzhsq28U/DrebJa+R7WQBG/2WNwJd/Zbz3eQRaFy13YuTjOa43RFPqupf3fem43zrfg64iprfXOHI6qO+8rifyadu91WD2xeRaOD/gItxvqVXuWUygAM4SfBMd7vzgE9wWk8A81TVfwbVqwEFXvdb1wMn0exwkw44vRUb3Ne/wmlRLnKT1N9UdXp1QRFJxGmhzPOry0DVPj6q/I8Pd6D+YZyWaLIb166GNhZAXf0A+D3wVxFZBvxWVRe4dXCliNzit7k4ah57Ec+6sCJL7amX/4HzbbSvqqbgdBcd0ZVDQdiG0+0BgIh4aPyPchvOH3O17jhXKjUJVS1U1VtUtSdwAfBbETnZfft1YISIHIvTvfRSgJsNZorr7+OcdANxjVv+VJzuyOruoOr/u7k4XVUnu68/xenKql729wegEHjBPdmCcwIvBjqoapr7k6KqgwFUdZvbFZUF3Ag8JSK94NCY1Zs43VY/C/SX97MNv6vQ3PGebL/378O5wvA495i9lprHbO26b7SuVHWBqp4PdMLp6prpVwd3+f3+aaqaUN1Kq2c/EckSSGRLxhk4PCAiA2h8/KOp/AcYLiLnuYPvN+OMPzTkZeBOEckQkY44J7wXmioYN44+biIrwBkwrgRwr4Z6w43hc7e7JhA7AZ+I9A4whiScCwnmBbj9ZJyT6G6cluOf/d90B4grgctwWgF7gb3AROomkDKcbrr2wHMiEqWqm91yfxORFBGJEpG+InKSG+8P3W4ugH04J9NKd5zodZx6nFyrpROo/wBDRWSie3zcQs3jIxmn5VAgIt2AX9f6/A6ccRH/8vXWlYi0E5ErRCRFVcuB/Xx36ftTwI0iMkpEPCKS5B4riQ3sJyJZAolsv8IZpN2P0xp5JdQ7VNUdwKXA33H+qPsAS3D+yOtzF84VPMtx7itYgNMl0VQE+AgowunbfkhVP/N7fzrOVVC1u68a5Pb5/x+wQET2uYPJjTkd+FRVywLcxXM4g7hbgRXAF/WUmQfsVNWt7vJcnO6bOldDueMcF+B803/aTaZX4Qzcr8RJPq/iDEKDM6i9UEQO4CSMG1V1E04r52z3p0Ccm0uLROT4AH8v/+PjrzjHR3dqdqv+EeeiiwJgNvCvWpv4C3CXW++/5PB1NQnYKCKFwBScLj3cbqyfAk+4v/8qt04a2k9E8tgDpUw4ud0mW4GLVfXTcMdTm9uK+BrIVNWiw5UPch9PAYtU9alQbN+YULFBdNPs3Ov85+NcIXQHzuXEX4U1qHq4/e+3Ai+FKnm4FuOMGxjTqlgCMeEwHngR56qWFcAFbjdKi+FehroF58qj74VyX6r6ZCi3b0yoWBeWMcaYoNggujHGmKBEVBfW0qVLfV6vN6jPlpaWEuxn2yKrj7qsTmqy+qirtdZJcXFx/ogRI+pcbh9RCcTr9TJgwICgPpubmxv0Z9siq4+6rE5qsvqoq7XWSU5Ozsb61lsXljHGmKBYAjHGGBMUSyDGGGOCYgnEGGNMUCyBGGOMCYolEGOMMUGxBGKMMSYolkAC8NX6PWzcG+hM28YYExki6kbCYP35nVzaUcZZJ4Q7EmOMaTmsBRKADgmx7DpQEe4wjDGmRbEEEoDM1HbkF1sCMcYYf5ZAApCZEk9BSRWlFZWHL2yMMRHCEkgAslLjAdhZ2KKeeWSMMWFlCSQAnd0Esr2wJMyRGGNMy2EJJADVLZBtBZZAjDGmmiWQAHROcRLIDksgxhhziCWQAKTEx+CN8VgLxBhj/FgCCYDH4yEjIYYdNgZijDGHWAIJUEZCtA2iG2OMH0sgAcpIiGG7dWEZY8whlkAClO52YVVV+cIdijHGtAiWQAKUkRhNRZWP/AN2M6ExxoAlkIClJzgTF+8osARijDFgCSRg6QnRAGwrOBjmSIwxpmWwBBKgjtUtELsSyxhjAEsgAUuNjyY6ym4mNMaYamF9IqGInAU8BEQDz6jqvbXe9wIzgBHAbuBSVd3gvjcY+AeQAlQBo1Q1ZGf36CgPnZO9di+IMca4wtYCEZFo4DHgbGAgcLmIDKxVbAqwV1X7AlOB+9zPxgAvADeo6rHAKUB5qGPunBpvXVjGGOMKZxfWaGCNqq5T1TJgJjCxVpmJwHT39WvAaSLiAc4EvlbVZQCqultVQ/60p6zUeOvCMsYYVzi7sLoCm/2W84AxDZVR1QoRKQDSgWMAn4i8D3QEZqrq/YfbYWlpKbm5uUEFW1JSQmxFMVv3FrNy5Uo8Hk9Q22krSkpKgq7LtsrqpCarj7raWp2EM4HUdwaufZt3Q2VigPHAKKAYmCMiOao6p7Eder1eBgwYEEys5ObmMrBnCm/mfkt2736kxMcGtZ22Ijc3N+i6bKusTmqy+qirtdZJTk5OvevD2YWVB3TzW84GtjZUxh33SAX2uOvnqmq+qhYD7wDDQx1wZqo9F8QYY6qFM4EsBPqJSC8RiQMuA2bXKjMbmOS+vhj4SFV9wPvAYBFJcBPLycDKUAecmWJPJjTGmGphSyCqWgH8HCcZ5AKzVHWFiNwtIue7xZ4F0kVkDXArcLv72b3A33GS0FJgsaq+HeqYs1LbAfZsdGOMgTDfB6Kq7+B0P/mvu9PvdQlwSQOffQHnUt5m0ynFC1gXljHGgN2JfkTiY6NpnxDLNmuBGGOMJZAjlZnazlogxhiDJZAjlpnitUF0Y4zBEsgRy0xtZ9OZGGMMlkCOWGZKPLsPlFFaEfKZU4wxpkWzBHKEOrtXYu0stCcTGmMimyWQI5Se5CSQPQfKwhyJMcaElyWQI5SRFAfA7gPWAjHGRDZLIEcow22B5BdZC8QYE9ksgRyh9OoWiCUQY0yEswRyhBLiYmgXG83uIuvCMsZENksgQUhPimO3DaIbYyKcJZAgpCd5ybcWiDEmwlkCCUJGYpyNgRhjIp4lkCA4XVjWAjHGRDZLIEFIT/Kyu6gMn6/2I9yNMSZyWAIJQnpiHBVVPgoPVoQ7FGOMCRtLIEE4dDOhdWMZYyKYJZAg2M2ExhhjCSQo6YlOC8RuJjTGRDJLIEGonlAx324mNMZEMEsgQWifWN2FZS0QY0zksgQShNjoKNonxNoYiDEmolkCCVJ6ktduJjTGRDRLIEFKT4yzZ4IYYyKaJZAgZSR5bQzEGBPRLIEEyaZ0N8ZEOksgQUpP9LKvuJzyyqpwh2KMMWFhCSRI1Xej77VWiDEmQlkCCdKhmwltIN0YE6EsgQQp3Z1Q0S7lNcZEqphw7lxEzgIeAqKBZ1T13lrve4EZwAhgN3Cpqm7we787sBL4k6r+rbniBucyXrAJFY0xkStsLRARiQYeA84GBgKXi8jAWsWmAHtVtS8wFbiv1vtTgXdDHWt9qlsg9mx0Y0ykCmcX1mhgjaquU9UyYCYwsVaZicB09/VrwGki4gEQkQuAdcCKZoq3hpT4GGKjPXYprzEmYoWzC6srsNlvOQ8Y01AZVa0QkQIgXUQOAr8FzgB+HegOS0tLyc3NDSrYkpKSOp9N8UaxJm8HubmR92jb+uoj0lmd1GT1UVdbq5NwJhBPPetqn4kbKnMXMFVVi0Qk4B16vV4GDBgQeIR+cnNz63w2My2fypj4oLfZmtVXH5HO6qQmq4+6Wmud5OTk1Ls+nAkkD+jmt5wNbG2gTJ6IxACpwB6clsrFInI/kAZUiUiJqj4a+rC/k27TmRhjIthhE4iIJAIHVbVKRI4B+gPvqmr5Ue57IdBPRHoBW4DLgCtqlZkNTALmAxcDH6mqDzjRL74/AUXNnTzAuRdk7c6i5t6tMca0CIEMos8D4kWkKzAHmAw8f7Q7VtUK4OfA+0AuMEtVV4jI3SJyvlvsWZwxjzXArcDtR7vfppThTunu80XeGIgxxgTSheVR1WIRmQI8oqr3i8iSpti5qr4DvFNr3Z1+r0uASw6zjT81RSzBSE+Mo6S8iuKyShK9Yb2lxhhjml0gLRCPiBwPXAm87a6zsyV+d6PbzYTGmAgUSAL5JXAH8IbbxdQb+Di0YbUO1RMq5tt0JsaYCHTYloSqzgXmAohIFJCvqr8IdWCtQUaitUCMMZHrsC0QEXlJRFLcq7FWAioit4U+tJavugVil/IaYyJRIF1YA1W1ELgAZ8C7O3B1SKNqJTq4Eyru2m8JxBgTeQJJILEiEouTQN507/+w61aB+Nhouqa1Y5XdC2KMiUCBJJB/ABuARGCeiPQACkMZVGsyODuVZZv3hTsMY4xpdoEMoj8MPOy3aqOITAhdSK3LkG5pvPvNdvYeKKO926VljDGRIJCpTFKBPwInuavmAncDBSGMq9UYnJ0KwLK8fZwincIcjTHGNJ9AurCmAfuBH7o/hcBzoQyqNTmuayoeD3ydZ/nUGBNZArmjvI+qXuS3fJeILA1VQK1NcnwsfTom8XWejYMYYyJLIC2QgyIyvnpBRMYBB0MXUuszODuVpZsLbFJFY0xECaQF8lNgujsW4sF5Hse1oQyqtRnaLY3XF29hW0EJXdLahTscY4xpFoFchbUUGCIiKe6yXcJby+DsNACWbd5nCcQYEzEaTCAicmsD6wFQ1b+HKKZWZ0BWMrHRHpblFXD2cVnhDscYY5pFYy2Q5GaLopXzxkQzICvFBtKNMRGlwQSiqnc1ZyCt3eDsVN5cspWqKh9RUZ5wh2OMMSEXyFVYJgBDstPYX1rBuvwD4Q7FGGOahSWQJjKk23cD6cYYEwksgTSRPh2TSIyLtnEQY0zECGQuLC9wEdDTv7yq3h26sFqf6CgPg7qmssymNDHGRIhAWiBvAhOBCuCA34+pZVj39qzYWkBxWUW4QzHGmJAL5E70bFU9K+SRtAHj+qbz5Ny1LFi/hwk2M68xpo0LpAXyhYgcF/JI2oBRPTsQFxPFZ6vzwx2KMcaEXCAtkPHAtSKyHijFmQ/Lp6qDQxpZKxQfG83onh34fI0lEGNM2xdIAjk75FG0IeP6ZnDfe9+yc38JnZLjwx2OMcaEzGG7sFR1I5AGnOf+pLnrTD1O7JcBYK0QY0ybd9gEIiI3Ay8CndyfF0TkplAH1loNzEqhfUIsn9o4iDGmjQukC2sKMEZVDwCIyH3AfOCRUAbWWkVFeTihbwafr8nH5/Ph8di8WMaYtimQq7A8QKXfcqW7zjTgxL4Z7CgsZc3OonCHYowxIRNIC+Q5YIGIvOEuXwA82xQ7F5GzgIeAaOAZVb231vteYAYwAtgNXKqqG0TkDOBeIA4oA25T1Y+aIqamMN4dB/l0dT79Otus+MaYtimQQfS/A5NxHmW7F5isqg8e7Y5FJBp4DOcqr4HA5SIysFaxKcBeVe0LTAXuc9fnA+ep6nHAJOCfRxtPU8pun0CvjEQ+s4F0Y0wb1mACqX6ErYh0ADYAL+CcqDe6647WaGCNqq5T1TJgJs6UKf4mAtPd168Bp4mIR1WXqOpWd/0KIN5trbQY4/qm8+W63ZRXVoU7FGOMCYnGurBeAs4FcgCf33qPu9z7KPfdFdjst5wHjGmojKpWiEgBkI7TAql2EbBEVUsPt8PS0lJyc3ODCrakpOSIPtszvpTiskpen7eUwZlt7znpR1ofkcDqpCarj7raWp009kTCc91/e4Vo3/UNxPuOpIyIHIvTrXVmIDv0er0MGDAg4AD95ebmHtFnu/Wu4JEFH/H+xkounRDcPluyI62PSGB1UpPVR12ttU5ycnLqXR/IfSBzAlkXhDygm99yNrC1oTIiEgOk4ozFICLZwBvANaq6tgniaVJJ3hh+cnJvPvp2Jzkb94Q7HGOMaXINtkBEJB5IADJEpD3ftQZSgC5NsO+FQD8R6QVsAS4DrqhVZjbOIPl84GLgI1X1iUga8DZwh6p+3gSxhMS1J/Rk2mcbuP89Zeb1Y+2eEGNMm9JYC+QnOOMf/d1/q3/exLl66qioagXwc+B9IBeYpaorRORuETnfLfYskC4ia4Bbgdvd9T8H+gJ/EJGl7k+Lmz89IS6Gn0/ow4L1e+yKLGNMm9PYGMhDwEMicpOqhuSuc1V9B3in1ro7/V6XAJfU87l7gHtCEVNTu3xMd57+dD1/fV8Z3zfDWiHGmDbjsDcSquojIjII516NeL/1M0IZWFvhjYnm5tP78ZvXvub9FTs4a1BmuEMyxpgmEcgg+h9x5r16BJgA3A+c3+iHTA0XDutK746J3PXWCrbsOxjucIwxpkkEMhfWxcBpwHZVnQwMAVrUTXstXUx0FI9cPoyikgqufnYBu4sOe8uKMca0eIEkkIOqWgVUuHen7+TobyKMOMd2SeWZSSPZsvcgk59fSFFpRbhDMsaYoxJIAlnkXjb7NM5VWIuBr0IaVRs1pnc6j14xnBVbC/nJPxdRVmHTnBhjWq9AJlP8maruU9UngTOASW5XlgnCGQM7c99Fg/l8zW7+/sGqcIdjjDFBa+xGwuGNvaeqi0MTUtt38YhsFm3Ywz/mreXkYzpyfJ/0cIdkjDFHrLHLeB9w/40HRgLLcO5GHwwsAMaHNrS27Q/nDmTB+j3cOmsp7918EqkJseEOyRhjjkiDXViqOkFVJwAbgeGqOlJVRwDDgDXNFWBbleiN4cFLh7Jrfym/+/dyfL7a80gaY0zLFsggen9VXV69oKrfAENDF1LkGNItjVvOOIa3v97G64u3hDscY4w5IoE80jZXRJ7BeaCUD7gKZ+4q0wRuOLkPc3UXf5q9gjG9O5DdPiHcIRljTEACaYFMxnnq383AL4GV7jrTBKKjPDzwwyFU+Xzc9urXVFVZV5YxpnUIZC6sEpznkU8NfTiRqVuHBO48byC//ddypn2+nutOtPs0jTEtX2OX8c5S1R+KyHLqPikQVR0c0sgizA9HduODlTu4/33lpGM6ckzn5HCHZIwxjWqsC+tm999zgfPq+TFNyOPx8H8XDibZG8Ots5ZSUWl3qRtjWrbGngeyzf13Y/OFE9k6Jnu5e+IgbnxpMTMXbuaqsT3CHZIxxjSosS6s/dTTdYVzM6FPVVNCFlUE+/5xmYzu1YGpH6zi/KFdSIm3GwyNMS1TYy0Q64QPA4/Hwx/OGcj5j33GYx+t4Y7vDwh3SMYYU69A7gMBwH3muP8TCTeFJCLDcdmpXDgsm+c+38CVY3rQPd3uDTHGtDyBPJHwfBFZDawH5gIbgHdDHFfE+81ZQnSUh/971+7ZNMa0TIHcSPi/wFhglar2wnk64echjcrQOSWeG07uw7vfbOer9XvCHY4xxtQRSAIpV9XdQJSIRKnqx9hcWM3i+pN60zHZy0Nz7LkhxpiWJ5AEsk9EkoB5wIsi8hBgz2NtBu3iovnxib34fM1ulmzaG+5wjDGmhkASyETgIHAL8B6wFruRsNlcOaYHaQmxPPaxzaBvjGlZGrsP5FHgJVX9wm/19NCHZPwlemOYfEIvpn64itxthQzIsttvjDEtQ2MtkNXAAyKyQUTuExEb9wiTa0/oSZI3xlohxpgWpbEnEj6kqscDJwN7gOdEJFdE7hSRY5otQkNqQixXje3B28u3sW5XUbjDMcYYIIAxEFXdqKr3qeow4ArgB9gDpZrdlPG9iIuO4olP1oY7FGOMAQK7kTBWRM4TkRdxbiBcBVwU8shMDR2TvVw8IpvZy7ZSVGoXwRljwq+xQfQzgMuBc4CvgJnA9ap6oJliM7X8YFhXXlywiTm5O5g4tGu4wzHGRLjG5sL6HfAS8GtVDcmt0CJyFvAQEA08o6r31nrfC8wARgC7gUtVdYP73h3AFKAS+IWqvh+KGFuS4d3bk5Uaz1vLtloCMcaEXWOD6BNU9ekQJo9o4DHgbGAgcLmIDKxVbAqwV1X74jxS9z73swOBy4BjgbOAx93ttWlRUR7OHZzF3FW7KCguD3c4xpgIF8iNhKEyGlijqutUtQyni2xirTIT+e7ek9eA00TE466fqaqlqroeWONur807b0gXyit9vL9ye7hDMcZEuICncw+BrsBmv+U8YExDZVS1QkQKgHR3/Ze1PnvYPp3S0lJyc4O7gKykpCTozzalGJ+PzKQYZn6xmuMSw3dJb0upj5bE6qQmq4+62lqdhDOBeOpZV/sJiA2VCeSzdXi9XgYMCO4BTbm5uUF/tqldtDmKJ+euo1O33qQnecMSQ0uqj5bC6qQmq4+6Wmud5OTk1Ls+nF1YeUA3v+VsYGtDZUQkBkjFuakxkM+2WecN6UJllY93v7FuLGNM+IQzgSwE+olILxGJwxkUn12rzGxgkvv6YuAjVfW56y8TEa+I9AL64VxqHBGkczJ9OyXx1rKIyZnGmBYobAlEVSuAnwPv49zZPktVV4jI3SJyvlvsWSBdRNYAtwK3u59dAcwCVuLMEHyjqlY29+8QLh6Ph/MGd+GrDXvYXlAS7nCMMREqnGMgqOo7wDu11t3p97oEuKSBz/4Z+HNIA2zBzhuSxdQPV/Gfr7dy3Ym9wx2OMSYChbMLyxyF3h2TGJydyr+Xbgl3KMaYCGUJpBWbOLQr32wpZM3O/eEOxRgTgSyBtGLnDckiygP/XmKD6caY5mcJpBXrlBzPuL4Z/HvpFny+w94GY4wxTcoSSCv3g2Fdydt7kJyNe8MdijEmwlgCaeXOPDaT+NgoG0w3xjQ7SyCtXJI3hjMHZvKfr7dRVlEV7nCMMRHEEkgbcMGwLuwrLmfeql3hDsUYE0EsgbQBJ/brSIfEOF7N2Xz4wsYY00QsgbQBsdFRXD66G/9ducPuCTHGNBtLIG3Ej8b1Ij4mmsc/XhvuUIwxEcISSBuRnuTlyjHdeXPZVjbuPhDucIwxEcASSBvy45N6Ex3l4cm51goxxoSeJZA2pHNKPJeO7MZrOXls3Xcw3OEYY9o4SyBtzE9O7o3PB0/NWxfuUIwxbZwlkDYmu30CFw7vystfbeKz1fnhDscY04ZZAmmDbj1D6JGewDXTFvDMp+tsokVjTEhYAmmDMlPjef1n4zhjYGfueTuXW2cto6Q8Yp74a4xpJpZA2qgkbwxPXDmCX51xDG8s2cLVzy6gsKQ83GEZY9oQSyBtWFSUh5tO68djVwxnyaZ9XPn0AvYcKAt3WMaYNsISSAQ4Z3AWT18zklU79nPpP+azs7Ak3CEZY9oASyARYkL/Tjw/eTRb9x3kkn/MZ3uBJRFjzNGxBBJBju+TzgvXjWF3URmXP/2ltUSMMUfFEkiEGda9PdN/NIqdhSVc/vSX7NpfGu6QjDGtlGiXRM4AABWuSURBVCWQCDSiRweemzyarftKuPKZL23aE2NMUCyBRKjRvTrw7LUj2bznIKc9MJdH5qy2e0WMMUfEEkgEO6FPBh/cehIT+nfkgQ9Wcfrf5/Lqos3sKw7+Ut8HP1zFhY9/TlVV3bvfH/1oNY9/suZoQjbGtCAx4Q7AhFd2+wQev3IEX6zN567ZK7ntta+JjvIwokd7zjo2k6uP70FsdGDfMwqKy3lq3jqKyyqZt3oXp0inQ+/tLCzhoTmr8Xg8XDaqOx0S40L1Kxljmom1QAzgtEbevflE/n3jOH56ch8KD5Zz939WctdbKwLexktfbaK4rJJkbwwz5m+s8d4LCzZRXumjrKKKVxbas9uNaQssgZhDoqI8DO2Wxq+/J7z3y5O44eQ+vPDlJv45f8NhP1taUclzn6/nxH4ZTB7fi49156EnI5ZWVPLSgo2c2r8TY3t34IUvN1JZTxeXMaZ1CUsXloh0AF4BegIbgB+q6t56yk0C/sddvEdVp4tIAvAq0AeoBN5S1dubI+5Ic9v3hDU79/Ont1bSu2MS4/pmsKOwhBnzN7Bw9XYe7dqLTinxAMxeupWd+0v52yVDkMxkHv94DS98uZHfnzOQ/yzbRn5RGZPH9WR/SQU/e3ExH3+7k9MHdg7vL2iMOSrhaoHcDsxR1X7AHHe5BjfJ/BEYA4wG/igi7d23/6aq/YFhwDgRObt5wo4s0VEeHrxsGH07JvGzFxdz88wljL/vIx7/ZC1Ltx3k0qe+ZFvBQXw+H09/uo7+mcmc2C+DzinxnDUok1cWbqa4rILnv9hA305JjO+bwRkDO9M5xcuMLzcePgBjTIsWrgQyEZjuvp4OXFBPme8BH6jqHrd18gFwlqoWq+rHAKpaBiwGspsh5oiU5I3hmUkjiY7y8OHKHVw5pgef/PoU7j0zi/z9pVz6jy95ccEmVu0o4scn9sbj8QAw6YSeFJZU8Id/r2D5lgKuPaEnHo+H2Ogorhjdg3mrdrE+/0CYfztjzNEI11VYnVV1G4CqbhPxu1znO10B/9HWPHfdISKSBpwHPBTITktLS8nNzQ0q4JKSkqA/2xY8dm4WsVEeEuOiKN65iV4p8L+nd+J/PtjO//z7G9ITounnLTxUR4k+H73bx/GvxXkkxUVxbELRofdGdqggJgoefmcJ149KD+ev1aQi/RipzeqjrrZWJyFLICLyIZBZz1u/D3ATnnrWHRp5FZEY4GXgYVUN6AHgXq+XAQMGBLj7mnJzc4P+bFuUm5vLBcMG0Ld3AdfPWMRNp/Vj8KDuNcpcX5TE7a8v54qxPRk2uGbdna0VfKg7GdynK8O6t2dgVgpxMa37mg47Rmqy+qirtdZJTk5OvetDlkBU9fSG3hORHSKS5bY+soCd9RTLA07xW84GPvFbfgpYraoPNkG4JkiDuqby+e2nHuq68nfBsK5sLyzhmuN71nnv5xP6snTzXu56ayUAcTFRnNa/Ez8+qTfDu7evU94Y0/KEqwtrNjAJuNf99816yrwP/MVv4PxM4A4AEbkHSAWuC32o5nDqSx4A8bHR/PL0Y+p9TzKT+fQ3p7Kt4CBLN+1jwfo9vL44j3e/2c7IHu257sRenDagc8A3MRpjml+4Esi9wCwRmQJsAi4BEJGRwA2qep2q7hGR/wUWup+5212XjdMN9i2wWEQAHlXVZ5r9tzBHLSu1HVnHtePs47K47XvCrEWbefaz9dzwwmI6Jnu5aHg2Fw3vSlFpBUs27WPp5n0kemP42Sl96NYh4dB2vly3m7veWkn3Du146LJhxMdG17u/qiofLy7YyIbdxfz++wOIiqo/+RljDi8sCURVdwOn1bN+EX6tClWdBkyrVSaP+sdHTCuX6I1h8rheXD22B5/oLmYu3MzTn67jyblrD5XJTIlnb3EZ/8rJ4+rje3DZqG488claXl+yhc4pXr7dXsi1z33FM5NGkeSteXjn7S3mtle/Zv663QB0SWvHlPG9mvV3NKYtsbmwTIsTEx3F6QM7c/rAzuwsLOH9lTvomBTH0G7tyUyNZ3tBCVM/WMVzn6/n2c/WExvt4ecT+nLjhL78d+V2bp21jKueWcD0yaNJ9EazZlcRn63O58EPVwNw74XH8WHuDu5771tO6pdBv87JDcZSVeWzVooxDbAEYlq0TinxXD22R411manx3HfxYKac2It3l2/n3CFZ9OmYBMDEoV2Jj43mppeWcOoDn1BUWkFpRRUAx/dO5/6LB9OtQwKnDejM9x6cxy2zlvLGz8bVGWvZtLuYBz5Q/rtiB49eMYzTBthd88bUZgnEtFrHdE7mmHpaD987NpPnJo/iuc830DM9geOyUzm2Syp9OiYeGvDvmOzlLz84jhteyOGROau59UyhpLySzXuKeXHBJl5csJHoKA+dkuO58aXFvHjdWEb0aBlXh23eU0yVz0eP9MRwh2IinCUQ0yaN65vBuL4ZjZY5a1AmFw3P5tGP1/Dyws2HHu8bHeXhhyO78cvT+xEd5eHiJ77gR88v5LUbjm+0u+tolFdWMWP+RoZ1T2vwMubleQU8OXct736zjeT4WP57y0l0duciMyYcLIGYiPbH8wdS5fMRFx1Fdvt2dOuQwLDuaTW+3f9zyhgufOILrpn2FfdeNJiEuGhio6OorPKxa38JO/eXsmt/KQcL9zG8Yhtd0tqRkRRHQlwM7WKjiY+NavBSZ4AdhSXc9NISvtqwh3ax0Uy7dhTH9/nuDv28vcX87o1vmLdqF8neGCad0JOXv9rEba99zfTJo+ps2+fzsXxLAa8uymN9/gEevnyYPX/FhIQlEBPRUuJjmXrp0EbLdOuQwPOTR3HZP75k0rSv6i3j8YDPByzaU+e92GgPp0gnLh6RzQTpVOOO+/lrd3PTy0s4UFrBPRcMYsb8DUx+/iumTRrFCX0zeH/Fdm57dRk+H/z2rP5cObY7KfGx9MpI5M43V/DCgk2Hxogqq3y8snAzM+Zv4Nvt+/HGRFHl83HzzCU8P3k00XYxgGlilkCMCcCxXVKZ86uTWZd/gPLKKioqfeCBTsleOiXHk54YR87yFSR16s7WfQfJLyqlpLyKg+WVbC8o4e3l2/hg5Q46JMbRt1MShQfL2Vdczo79JfTOSOTlH4+hX+dkzhqUyZVPL+BH0xdy5sBMZi/bynFdU3n0imE1WkVXj+3BByt38Je3cxnfN4PSikpu/9dylm7ex+DsVO65YBDnDenCO8u3ccfry3l4zmpuOaP+mzqNCZYlEGMC1Ckl/tDzT+qTFBfNgKwUBmSl1Hnvf84ZwKer8/nX4jx2FpbSvUMCg7Nj6ZqWwJQTex26ZyUjyctLPx7DFU8vYPayrfxoXC9+e7bgjal5Y6TH4+GvFw/hew/O46pnFrCjsISUdrE8eOlQJg7tcqhb67JR3Vi0YS8Pf7SaYd3Tajxm2LRMyzbvo2dGIqntYsMdymFZAjGmGcRERzGhfycm9D/8CTw9ycurPz2eTbuLGdQ1tcFymanx/PkHg/jFy0u4cHg2v//+ANrXGuvweDzcc8EgVmwt4JevLOWvFw+hf2YyXdPaUeXzkbttPwvW72bl1kJG9erAuYOzSI5v+SeuturDlTu4bsYihndP45WfHB/wVD7fbCngjSVbyEqNp1/nZI7pnERmSnyjY29NwRKIMS1QSnxso8mj2rmDuzBBOpHobfhPuV1cNE9eNYILHv+cH89YBEB8bBQxUVEUlVYAkJYQy+tLtnDXWys4e1AWQ7JT2V9SQWFJOZVVcNIxzlVt/ie0goPlrNxayPr8A6zPLyJv70HG9k7nkpHZJMQ17anF5/OxdlcR3TskNjhrs8/n44u1u5n22Xr6ZyVz6xlyVOM+pRWVLM8rYMH6PWzeU8wp0olT+3cKatbo9fkH+Of8jUzIqmywzNpdRdzyylK6pMazeNM+7n/vW35/zsBGt7trfyl/e1+ZlbOZaI+HCr9HRV80PJu/XTI4pEnEEogxrVxjyaNaz4xEPv3NBHT7ftbsLGL1ziLKKqoY2bM9Y3ql0znFy7K8Al5dtJnZy7byxpItgJNofD6Y9vl60hJiOdN9DPHiTftYs7Po0Pa9MVFkJHl595vtTP1wFdeM7UHvdiXsXLWL3UWl7CsuJzrKeaBYbLSHPQfKWJ9/gHX5B9hRWEJMlAdvTDTe2CgGd03l7OOyGNWzAz6fj7eXb+OJT9by7fb9HNc1lYcuG0pv98ZRgIrKKv67cgdPfLKW5VsKSI6PYc63O8ndtp+HLhvaYIvK5/Ox50AZhSUV7C8pZ8+BMtbuOsDqHftZtWM/K7YWHroJNckbw8yFm2mfEMt5Q5ykPahrKh2TvYet+zeXbuF3ry/nQFkl76fE8vox/epcfr2/pJzrZywiLiaKV396Ak9+spanP13PqJ4dOPPY756KsaOwhFU79rNu1wHW7CzijSVbKCmv5LrxvbjptH6UV1SxakcR732zjenzN9KnUyI/O6XvYWMMlsfn8x2+VBuRm5vrs+eBNA2rj7raSp2UVlRSVFJBcnwscTFRlFZU8umq/EMXAsRGexjWvT3DuqUxuFsafTom0iW1HVFRHhZt2MM/5q3jw9wdHO7Ukp4YR6+MRLqktaPS56O0vIoDpRUs3rSX0ooqMpLiaBcXzeY9B+nbKYnzh3Rh2ufrKS2v4q7zj+WU/h2ZtXAzLy3YxNaCEnqmJ/CTk/vwg2FdeS0njz/OXkHvjESeumYkHRLiKHSTxLK8fXy5bjcL1u1h94GyOnG1T4jlmM7JHNsllTG9OzCqZwdS4mMOjWH9d+UOytzE0jnFS/cOCRSVVrK/pJwDpRX07ZTEmF7pjOrVgXeXb2Pmws2M6NGeKeN78atXlpCVlsDM68ceGk+rqvJxwws5zPl2Jy9MGcPxfdIprajk4ifms3H3AV75yfF8nbeP13LyWLhh76E4E+OiGdc3g9+e3f/QTAzVfD4fN728hLeXb+PZSSM5tf/RzaSQk5OTM2LEiJG111sCCfyzbeLk0FSsPuqKhDqprPIR5Wl4Cv9q6/MP8HFOLoOlN+lJXtLaxVLl81Fe6aOsoorUdrGkJtTfMjhQWsEnuot3vtnGvuIyrjm+J2cM6ExUlIftBSXc8spS5q/bTZQHqnwwvm8GV43twRkDO9fosvpiTT4/fXExBQfL6+yja1o7xvZOZ1DXFNISYkn2OvH0TE8kIymu0d+vqLSCFVsK+GZrId9sKWDrvoMkx8eQEh+LNzaalduc9ZVud9LPTunDLWccQ2x0FK/NXcKdc3aQmRrP9Sf25qv1e/hi7W62F5Zw57kD+ZHf5J6bdhdzziOfsr/E6Wbs0zGRC4dnM7x7e/p0TKRjsrfROA+WVXLxk1+waXcxb9x4An07BX8TrCUQLIE0JauPuqxOagpVfVRW+Xjhy41sLyzhkhHZNbqzatu0u5i3vt5Ku9ho5yTfLpaBWSk1HgUQCtUtqbR2cRyX/d1YVm5uLvvjO3Ptc19RXFZJh8Q4ju+TzhkDOte4eq7a52vymZO7k/OGZDG0W9oRj2ds2XeQ8x/5jE4p8bx784lB/z4NJRAbAzHGtCrRUR4mndAzoLLd0xO4cULoxgAakuiN4cR+Het9b3SvDnz0q1PYd7CMYzolNzrbcyBT8jSma1o7np88mnmrdwW9jcZYAjHGmGaWmRpPZmrzzGN2XHZqjVZQU7LnhRpjjAmKJRBjjDFBsQRijDEmKJZAjDHGBMUSiDHGmKBYAjHGGBMUSyDGGGOCYgnEGGNMUCJqKpOcnJxdwMZwx2GMMa1MjxEjRtS5tT6iEogxxpimY11YxhhjgmIJxBhjTFAsgRhjjAmKJRBjjDFBsQRijDEmKJZAjDHGBMUeKHUYInIW8BAQDTyjqveGOaRmJyLdgBlAJlAFPKWqD4lIB+AVoCewAfihqu4NV5zNTUSigUXAFlU9V0R6ATOBDsBi4GpVLQtnjM1JRNKAZ4BBgA/4EaBE6DEiIrcA1+HUxXJgMpBFGzpGrAXSCPcE8RhwNjAQuFxEBoY3qrCoAH6lqgOAscCNbj3cDsxR1X7AHHc5ktwM5Pot3wdMdetjLzAlLFGFz0PAe6raHxiCUzcReYyISFfgF8BIVR2E8wX0MtrYMWIJpHGjgTWqus79ljATmBjmmJqdqm5T1cXu6/04J4auOHUx3S02HbggPBE2PxHJBs7B+caNiHiAU4HX3CKRVh8pwEnAswCqWqaq+4jgYwSnh6ediMQACcA22tgxYgmkcV2BzX7Lee66iCUiPYFhwAKgs6puAyfJAJ3CGFpzexD4DU6XHkA6sE9VK9zlSDtWegO7gOdEZImIPCMiiUToMaKqW4C/AZtwEkcBkEMbO0YsgTTOU8+6iJ37RUSSgH8Bv1TVwnDHEy4ici6wU1Vz/FZH+rESAwwHnlDVYcABIqS7qj4i0h6n9dUL6AIk4nSF19aqjxFLII3LA7r5LWcDW8MUS1iJSCxO8nhRVV93V+8QkSz3/SxgZ7jia2bjgPNFZANOt+apOC2SNLe7AiLvWMkD8lR1gbv8Gk5CidRj5HRgvaruUtVy4HXgBNrYMWIJpHELgX4i0ktE4nAGwWaHOaZm5/bvPwvkqurf/d6aDUxyX08C3mzu2MJBVe9Q1WxV7YlzTHykqlcCHwMXu8Uipj4AVHU7sFlExF11GrCSCD1GcLquxopIgvv3U10fbeoYsdl4D0NEvo/z7TIamKaqfw5zSM1ORMYDn+Jciljd5/87nHGQWUB3nD+YS1R1T1iCDBMROQX4tXsZb2++u0RzCXCVqpaGM77mJCJDcS4qiAPW4Vy2GkWEHiMichdwKc5VjEtwLuntShs6RiyBGGOMCYp1YRljjAmKJRBjjDFBsQRijDEmKJZAjDHGBMUSiDHGmKDYbLzGNCERqcS53LnazKaawdmdRuY/7uR8xoSdJRBjmtZBVR0a7iCMaQ6WQIxpBu60J68AE9xVV6jqGhHpAUwDOuJMRjhZVTeJSGfgSZxJCgF+ijPtRbSIPI0zLcYWYKKqHmy2X8QYPzYGYkzTaiciS/1+LvV7r1BVRwOP4sxugPt6hqoOBl4EHnbXPwzMVdUhOHNKrXDX9wMeU9VjgX3ARSH+fYxpkLVAjGlajXVhvez371T39fHAhe7rfwL3u69PBa4BUNVKoMCd4XW9qi51y+TgPOnPmLCwFogxzcfXwOuGytTHf96kSuxLoAkjSyDGNJ9L/f6d777+AmdGX4Argc/c13Nwxj0QkWj3iX/GtCj27cWYptVORJb6Lb+nqtUPVvKKyAKcL26Xu+t+AUwTkdtwB9Hd9TcDT4nIFJyWxk9xnmxnTIths/Ea0wzcq7BGqmp+mEMxpslYF5YxxpigWAvEGGNMUKwFYowxJiiWQIwxxgTFEogxxpigWAIxxhgTFEsgxhhjgvL/hOb2eVXSglUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "training_val_losses = training_val_losses[:-patience] # plot only until early stopping\n",
    "plt.plot(range(len(training_val_losses)), training_val_losses)\n",
    "plt.ylabel('Validation loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.title(f'Training on \"{dataset_name}\" dataset')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
